# -*- coding:utf-8 -*-
# --author： jingfeng 
# time: 2019/2/13

from selenium import webdriver  # 浏览器驱动对象
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup
import re
import requests
import os


def get_urllist():
    browser = webdriver.Chrome()
    wait = WebDriverWait(browser, 30)
    browser.get('https://book.qidian.com/info/3684254#Catalog')
    html = browser.page_source
    soup = BeautifulSoup(html, 'html5lib')
    li = soup.find_all('div', class_="volume")
    browser.close()

    for i in li[1:]:
        big_title = i.find('h3').get_text()  #
        big_title = re.sub('\s+', '', big_title)
        try:
            big_title = re.sub('分卷阅读', '', big_title)
        except:
            big_title = big_title
        print(big_title)
        os.makedirs(big_title)
        lists = i.find_all('li')
        for list in lists:
            title = list.find('a').get_text()
            href = list.find('a')['href']
            print(title)  # 章节
            print(href)  # 章节链接

            get_content(href, title, big_title)


def get_content(url, title, big_title):
    """获取正文内容"""

    url = 'https:' + url
    response = requests.get(url=url)
    html = response.text
    soup = BeautifulSoup(html, 'html5lib')
    list = soup.find('div', class_="read-content j_readContent").find_all('p')
    content = ''
    for info in list:
        content += info.get_text()
        content += '\n'

    file = open(big_title + '/' + title + '.txt', 'w', encoding='utf-8')
    file.write(content)
    file.flush()
    print(title)


get_urllist()
